package com.cgx.webmagic;

import com.cgx.common.BeanUtil;
import com.cgx.dao.AreaDao;
import com.cgx.entity.Area;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.stereotype.Service;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import javax.annotation.PostConstruct;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author chenguanxin
 * @date 2020-07-16 21:47
 */
@Component
public class MyPageProcessor implements PageProcessor {

    @Autowired
    private AreaDao areaDao;


    private String baseUrl = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2019/";

    @Override
    public void process(Page page) {
        System.out.println(page.getHtml().toString());
        //  判断一级省
        Boolean provinceFlag = provinceSpider(page);
        if (provinceFlag) {
            return;
        }

        //  判断二级市
        Boolean cityFlag = citySpider(page);
        if (cityFlag) {
            return;
        }

        //  判断三级区
        Boolean countyFlag = countySpider(page);
        if (countyFlag) {
            return;
        }

        //  判断四级街道
        Boolean townFlag = townSpider(page);
        if (townFlag) {
            return;
        }

        //  判断五级村委
        Boolean villageFlag = villageSpider(page);
        if (villageFlag) {
            return;
        }

    }

    @Override
    public Site getSite() {
        return Site.me()
                .setSleepTime(100)// 设置爬取间隔时间
                .setTimeOut(10000)// 设置爬取超时时间
                .setRetryTimes(100);// 设置重试次数
    }

    public Boolean provinceSpider(Page page) {
        Boolean flag = false;
        //  判断一级省
        String provinceHtmlText = page.getHtml().xpath("//*[@class='provincetr']").nodes().toString();
        if ("[]".equals(provinceHtmlText)) {
            return false;
        }

        System.out.println(provinceHtmlText);
        // <td><a href="11.html">北京市<br></a></td>
        String provinceRegular = "<td><a href=\"(.*?)\">(.*?)<br></a></td>";
        Pattern provincePattern = Pattern.compile(provinceRegular);
        Matcher provinceMatcher = provincePattern.matcher(provinceHtmlText);
        List<Map<String, String>> provinceList = new ArrayList<>();
        System.out.println(provinceList);
        while (provinceMatcher.find( )) {
            Map<String, String> map = new HashMap<>();
            map.put("html", provinceMatcher.group(1));
            map.put("name", provinceMatcher.group(2));
            provinceList.add(map);
            System.out.println("Found value: " + provinceMatcher.group(0) );
            System.out.println("Found value: " + provinceMatcher.group(1) );
            System.out.println("Found value: " + provinceMatcher.group(2) );
        }
        if (provinceList.size() > 0) {
            flag = true;
            for (Map<String, String> province : provinceList){
//                {name=北京市, html=11.html}
                Area area = new Area();
                String html = province.get("html");
                String name = province.get("name");
                String id = html.replaceAll(".html", "");
                area.setId(id);
                area.setParentId("0");
                area.setParentIds("0," + id);
                area.setAdcode(id);
                area.setName(name);
                area.setType(2);
                area.setCreateDate(new Date());
                areaDao.save(area);
                page.addTargetRequest(baseUrl + html);
            }
        }
        return flag;
    }

    public Boolean citySpider(Page page) {
        Boolean flag = false;
        // 判断二级市
        String cityHtmlText = page.getHtml().xpath("//*[@class='citytr']").nodes().toString();
        if ("[]".equals(cityHtmlText)) {
            return false;
        }

        System.out.println(cityHtmlText);
//        <td><a href="44/4401.html">440100000000</a></td>
//        <td><a href="44/4401.html">广州市</a></td>
        String cityRegular = "<td><a href=\"(.*?)\">(.*?)</a></td>\n" +
                " <td><a href=\"(.*?)\">(.*?)</a></td>";
        Pattern cityPattern = Pattern.compile(cityRegular);
        Matcher cityMatcher = cityPattern.matcher(cityHtmlText);
        List<Map<String, String>> cityList = new ArrayList<>();
        while (cityMatcher.find()) {
            Map<String, String> map = new HashMap<>();
            map.put("html", cityMatcher.group(1));
            map.put("id", cityMatcher.group(2));
            map.put("name", cityMatcher.group(4));
            cityList.add(map);
            System.out.println("Found value: " + cityMatcher.group(0) );
            System.out.println("Found value: " + cityMatcher.group(1) );
            System.out.println("Found value: " + cityMatcher.group(2) );
            System.out.println("Found value: " + cityMatcher.group(3) );
            System.out.println("Found value: " + cityMatcher.group(4) );
        }
        System.out.println(cityList);
        if (cityList.size() > 0) {
            flag = true;
            for (Map<String, String> city : cityList) {
//                {name=广州市, html=44/4401.html, id=440100000000}
                String id = city.get("id");
                String html = city.get("html");
                String name = city.get("name");
                String parentId = html.split("/")[0];
                Area area = areaDao.findById(parentId).get();
                area.setId(id);
                area.setParentId(parentId);
                area.setParentIds(area.getParentIds() + "," + id);
                area.setAdcode(id);
                area.setName(name);
                area.setType(3);
                area.setCreateDate(new Date());
                areaDao.save(area);
                page.addTargetRequest(baseUrl + html);
            }
        }
        return flag;
    }

    public Boolean countySpider(Page page) {
        Boolean flag = false;
//        判断三级区
        String countyHtmlText = page.getHtml().xpath("//*[@class='countytr']").nodes().toString();
        if ("[]".equals(countyHtmlText)) {
            return false;
        }

        System.out.println(countyHtmlText);
//        <td><a href="01/440118.html">440118000000</a></td>
//        <td><a href="01/440118.html">增城区</a></td>
        String countyRegular = "<td><a href=\"(.*?)\">(.*?)</a></td>\n" +
                " <td><a href=\"(.*?)\">(.*?)</a></td>";
        Pattern countyPattern = Pattern.compile(countyRegular);
        Matcher countyMatcher = countyPattern.matcher(countyHtmlText);
        List<Map<String, String>> countyList = new ArrayList<>();
        while (countyMatcher.find()) {
            Map<String, String> map = new HashMap<>();
            map.put("html", countyMatcher.group(1));
            map.put("id", countyMatcher.group(2));
            map.put("name", countyMatcher.group(4));
            countyList.add(map);
            System.out.println("Found value: " + countyMatcher.group(0) );
            System.out.println("Found value: " + countyMatcher.group(1) );
            System.out.println("Found value: " + countyMatcher.group(2) );
            System.out.println("Found value: " + countyMatcher.group(3) );
            System.out.println("Found value: " + countyMatcher.group(4) );
        }
        System.out.println(countyList);
        if (countyList.size() > 0) {
            flag = true;
            for (Map<String, String> county : countyList) {
//                广州id=440100000000
//                {name=荔湾区, html=01/440103.html, id=440103000000}
                String id = county.get("id");
                String html = county.get("html");
                String name = county.get("name");
                String parentId = id.substring(0, 4) + "00000000";
                Area area = areaDao.findById(parentId).get();
                area.setId(id);
                area.setParentId(parentId);
                area.setParentIds(area.getParentIds() + "," + id);
                area.setAdcode(id);
                area.setName(name);
                area.setType(4);
                area.setCreateDate(new Date());
                areaDao.save(area);
                page.addTargetRequest(baseUrl + id.substring(0,2) + "/" + html);
            }
        }
        return flag;
    }

    public Boolean townSpider(Page page) {
        Boolean flag = false;
        //  判断四级街道
        String townHtmlText = page.getHtml().xpath("//*[@class='towntr']").nodes().toString();
        if ("[]".equals(townHtmlText)) {
            return false;
        }

        System.out.println(townHtmlText);
//        <td><a href="18/440118003.html">440118003000</a></td>
//        <td><a href="18/440118003.html">朱村街道</a></td>
        String townRegular = "<td><a href=\"(.*?)\">(.*?)</a></td>\n" +
                " <td><a href=\"(.*?)\">(.*?)</a></td>";
        Pattern townPattern = Pattern.compile(townRegular);
        Matcher townMatcher = townPattern.matcher(townHtmlText);
        List<Map<String, String>> townList = new ArrayList<>();
        while (townMatcher.find()) {
            Map<String, String> map = new HashMap<>();
            map.put("html", townMatcher.group(1));
            map.put("id", townMatcher.group(2));
            map.put("name", townMatcher.group(4));
            townList.add(map);
            System.out.println("Found value: " + townMatcher.group(0) );
            System.out.println("Found value: " + townMatcher.group(1) );
            System.out.println("Found value: " + townMatcher.group(2) );
            System.out.println("Found value: " + townMatcher.group(3) );
            System.out.println("Found value: " + townMatcher.group(4) );
        }
        System.out.println(townList);
        if (townList.size() > 0) {
            flag = true;
            for (Map<String, String> town : townList) {
//                增城=440118000000
//                {name=荔城街道, html=18/440118001.html, id=440118001000}
                String id = town.get("id");
                String html = town.get("html");
                String name = town.get("name");
                String parentId = id.substring(0, 6) + "000000";
                Area area = areaDao.findById(parentId).get();
                area.setId(id);
                area.setParentId(parentId);
                area.setParentIds(area.getParentIds() + "," + id);
                area.setAdcode(id);
                area.setName(name);
                area.setType(5);
                area.setCreateDate(new Date());
                areaDao.save(area);
                page.addTargetRequest(baseUrl + id.substring(0,2) + "/" + id.substring(2,4) + "/" + html);
                //  兼容例如东莞这种没有区的城市
                page.addTargetRequest(baseUrl + id.substring(0,2) + "/" + html);
            }
        }
        return flag;
    }

    public Boolean villageSpider(Page page) {
        Boolean flag = false;
        //  判断五级村委
        String villageHtmlText = page.getHtml().xpath("//*[@class='villagetr']").nodes().toString();
        if ("[]".equals(villageHtmlText)) {
            return false;
        }

        System.out.println(villageHtmlText);
//        <td>440118001223</td>
//        <td>220</td>
//        <td>群爱村委会</td>
        String villageRegular = "<td>(.*?)</td>\n" +
                " <td>(.*?)</td>\n" +
                " <td>(.*?)</td>";
        Pattern villagePattern = Pattern.compile(villageRegular);
        Matcher villageMatcher = villagePattern.matcher(villageHtmlText);
        List<Map<String, String>> villageList = new ArrayList<>();
        while (villageMatcher.find()) {
            Map<String, String> map = new HashMap<>();
            map.put("id", villageMatcher.group(1));
            map.put("name", villageMatcher.group(3));
            villageList.add(map);
            System.out.println("Found value: " + villageMatcher.group(0) );
            System.out.println("Found value: " + villageMatcher.group(1) );
            System.out.println("Found value: " + villageMatcher.group(2) );
            System.out.println("Found value: " + villageMatcher.group(3) );
        }
        System.out.println(villageList);
        if (villageList.size() > 0) {
            flag = true;
            for (Map<String, String> village : villageList) {
//                荔城街道=440118001000
//                {name=湘江社区居委会, id=440118001001
                String id = village.get("id");
                String name = village.get("name");
                String parentId = id.substring(0, 9) + "000";
                Area area = areaDao.findById(parentId).get();
                area.setId(id);
                area.setParentId(parentId);
                area.setParentIds(area.getParentIds() + "," + id);
                area.setAdcode(id);
                area.setName(name);
                area.setType(6);
                area.setCreateDate(new Date());
                areaDao.save(area);
            }
        }
        return flag;
    }

}
